﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Data;
using System.Data.SqlClient;
using WatiN.Core;
using System.Configuration;
using WatiN.Core.Native.InternetExplorer;
using System.IO;
using log4net;
using log4net.Config;
using HtmlAgilityPack;
using System.Net;

namespace PlayerStat
{
    class ScrapeScot : ScrapeSite
    {
        WebClient wc = new WebClient();
        static int tempNumber = 1;
        ILog logger;
        private int leagueID = -1;
        private string match_url_template = @"http://www.scottishfootballleague.com/football/{0}/results/archive/{1}-{2}";
        private string league = string.Empty;

        public ScrapeScot()
        {
            logger = LogManager.GetLogger("scrape");
            if (logger == null)
            {
                throw new Exception("Null Logger");
            }
        }

        public void SetLeague(int nLeagueID)
        {
            leagueID = nLeagueID;
            switch (leagueID)
            {
                case 20:
                    league = "first";
                    break;
                case 22:
                    league = "second";
                    break;
                case 23:
                    league = "third";
                    break;
                case 24:
                    league = "league-cup";
                    break;
                default:
                    throw new Exception("Wrong League Number");
            }
        }

        public bool Scrape()
        {
            logger.Debug("#Scrape Scotland Leauge# Start");
            HtmlDocument mainDoc = new HtmlDocument();
            HtmlDocument statDoc = new HtmlDocument();

            try
            {
                EventInit evnt = new EventInit(leagueID,Leagues.ScotlandID.ToString());
                string date = string.Empty;
                int year = 2009;
                string url = string.Empty;

                while (evnt.GetEventYear(leagueID, out date, out year))
                {
                    logger.DebugFormat("Go to season {0}-{1}", year, year + 1);
                    url = GetMatchURL(year);
                    logger.Debug("Download match list.");
                    wc.DownloadFile(url,@"C:\Scrape\MainIE.html");
                    logger.Debug("Goto match list.");
                    mainDoc.Load(@"C:\Scrape\MainIE.html",Encoding.GetEncoding("ISO-8859-1"));

                    HtmlNode div = mainDoc.DocumentNode.SelectSingleNode("//div[@id='copy']");

                    if (div == null)
                    {
                        logger.ErrorFormat("The season {0}-{1} data are didn't found.", year, year + 1);
                        continue;
                    }

                    HtmlNodeCollection tables = div.SelectNodes(".//table[@id='comp121_results']");

                    if (tables == null)
                    {
                        logger.ErrorFormat("The season {0}-{1} data are didn't found.", year, year + 1);
                        continue;
                    }

                    logger.DebugFormat("There are {0} matches", tables.Count);

                    foreach (HtmlNode tbl in tables)
                    {
                        DateTime dt = Translate.TransDateTime(tbl.SelectSingleNode("./tr[1]/th").InnerText);

                        if (!string.IsNullOrEmpty(date) && dt > DateTime.Parse(date))
                        {
                            continue;
                        }
                        else
                        {
                            HtmlNodeCollection matches = tbl.SelectNodes("./tr[@class='match']");
                            if (matches != null)
                            {
                                foreach (HtmlNode tr in matches)
                                {
                                    HtmlNode tc = tr.SelectSingleNode("./td[@class='cell4']");
                                    if (tc == null)
                                    {
                                        throw new Exception("Can not find URL of match details");
                                    }
                                    else
                                    {
                                        MatchData md = new MatchData();
                                        md.matchTime = dt;
                                        md.LeagueID = leagueID;

                                        if (!GetMatchData(tr, ref md))
                                        {
                                            logger.Error("Failed to GetMatchData.");
                                            return false;
                                        }
                                        logger.DebugFormat("Scrape the match({0} : {1} ) data.", new string[] { md.home, md.away });
                                        if (md.ExistInDB())
                                        {
                                            logger.Debug("Data is already scraped into DB.");
                                            continue;
                                        }
                                        logger.Debug("Download match statistic page.");
                                        string stat_url = @"http://www.scottishfootballleague.com" + tc.SelectSingleNode("./a[1]").Attributes["href"].Value;
                                        wc.DownloadFile(stat_url, @"C:\Scrape\StatIE.html");
                                        logger.Debug("Goto match statistic page.");
                                        statDoc.Load(@"C:\Scrape\StatIE.html", Encoding.GetEncoding("ISO-8859-1"));
                                        
                                        if (!GetMatchStat(statDoc.DocumentNode, ref md))
                                        {
                                            logger.ErrorFormat("Error to get statistic in {0} vs {1}", md.home, md.away);
                                            continue;
                                        }
                                        
                                        HtmlNode row = tr;
                                        if (!GetGoals(ref row, ref md))
                                        {
                                            logger.Error("Failed to GetGoals");
                                            return false;
                                        }
                                        if (!GetBookings(ref row, true, ref md))
                                        {
                                            logger.Error("Failed to GetBookings(Yellow Card)");
                                            return false;
                                        }
                                        if (!GetBookings(ref row, false, ref md))
                                        {
                                            logger.Error("Failed to GetBookings(Red Card)");
                                            return false;
                                        }
                                        logger.Debug("Save match to DB.");
                                        md.SaveToDB();                                   
                                    }
                                }
                            }
                        }
                    }
                }
                logger.Debug("#Scrape Scotland Leauge# End");
                return true;
            }
            catch (Exception ex)
            {
                logger.Error("#Scrape Scotland League# Failed: Exception.");
                logger.Error(ex.ToString());
                return false;
            }
            finally
            {
                logger.Debug("#Scrape Scotland League# Finally");
                wc.Dispose();
                mainDoc = null;
                statDoc = null;
            }
        }

        public string GetMatchURL(int year)
        {
            return string.Format(match_url_template, league, year, year + 1);
        }

        public bool GetMatchStat(HtmlNode doc, ref MatchData md)
        {
            HtmlNode table = doc.SelectSingleNode("//table[@class='full_results match_result']");
            if (table == null)
            {
                logger.Error("Not find the match_result data.");
                return false;
            }
            if (!GetPlayers(table, ref md))
            {
                logger.Error("Failed to GetPlayers.");
                return false;
            }
            return true;
        }

        public bool GetMatchData(HtmlNode tr, ref MatchData md)
        {
            try
            {
                if (tr == null)
                {
                    logger.Error("Match score is not found");
                    return false;
                }
                HtmlNode tc;
                tc = tr.SelectSingleNode("./td[@class='cell1']");//Home
                md.home = HtmlEntity.DeEntitize(tc.InnerText);
                //md.homeID = GetTeamID(tc.SelectSingleNode("./a[1]").Attributes["href"].Value);
                tc = tr.SelectSingleNode("./td[@class='cell2']");//Score
                md.match.homeScore = int.Parse(tc.SelectSingleNode("./span[1]").InnerText);
                md.match.awayScore = int.Parse(tc.SelectSingleNode("./span[2]").InnerText);
                tc = tr.SelectSingleNode("./td[@class='cell3']");//Away
                md.away = HtmlEntity.DeEntitize(tc.InnerText);
                //md.awayID = GetTeamID(tc.SelectSingleNode("./a[1]").Attributes["href"].Value);
                return true;
            }
            catch (Exception ex)
            {
                logger.Error("GetMatchData# Failed: exception");
                logger.Error(ex.ToString());
                return false;
            }
        }

        public int GetTeamID(string str)
        {
            // Format: "http:///www.scottishfootballleague.com/club/TeamName/"
            string temp = str.Substring(0, str.Length - 1);
            temp = temp.Substring(temp.LastIndexOf('/') + 1);
            return temp.GetHashCode();
        }

        public bool GetPlayers(HtmlNode tb, ref MatchData md)
        {
            try
            {
                var result = from t in tb.ChildNodes
                             where t.InnerText == "Teams"
                             select t;
                if (result.Count() != 1)
                {
                    logger.Error("Start players' data are not Found");
                    return false;
                }
                HtmlNode tr = result.First().NextSibling ;
                HtmlNodeCollection tables = tr.SelectNodes(".//table");
                if (tables != null)
                {
                    foreach (HtmlNode t in tables)
                    {
                        HtmlNode td = t.SelectSingleNode("./tr[1]/td[1]");
                        if ( td != null && td.InnerText == md.home)
                        {
                            // Home players
                            for (int i = 1; i < t.ChildNodes.Count; i++)
                            {
                                int number;
                                
                                number = GenPlayerNumber();
                                HtmlNode playercell = t.ChildNodes[i].ChildNodes[2];
                                PlayerStat ps = new PlayerStat();
                                if (!GetPlayerData(playercell,false, ref ps))
                                {
                                    logger.Error("Failed to get player data.");
                                }
                                md.homeplayers.Add(number, ps);
                            }
                        }
                        else if (td != null && td.InnerText == md.away)
                        {
                            // Away players
                            for (int i = 1; i < t.ChildNodes.Count; i++)
                            {
                                int number;

                                number = GenPlayerNumber();
                                HtmlNode playercell = t.ChildNodes[i].ChildNodes[2];
                                PlayerStat ps = new PlayerStat();
                                if (!GetPlayerData(playercell, false, ref ps))
                                {
                                    logger.Error("Failed to get player data.");
                                }
                                md.awayplayers.Add(number, ps);
                            }
                        }
                        else
                        {
                            continue;
                        }
                    }
                }
                result = from t in tb.ChildNodes
                         where t.InnerText == "Substitutes"
                             select t;
                if (result.Count() != 1)
                {
                    logger.Error("Start players' data are not Found");
                    return false;
                }
                tr = result.First().NextSibling;
                tables = tr.SelectNodes(".//table");
                if (tables != null || tables.Count == 2)
                {
                    HtmlNode tab = tables[0];
                    // Home sub-players
                    for (int i = 0; i < tab.ChildNodes.Count; i++)
                    {
                        HtmlNode playercell = tab.ChildNodes[i].ChildNodes[2];

                        if (IsSwitchOut(playercell.InnerText))
                        {
                            // IsSwitchOut used to check if has time data of the player
                            // Sub without the time data, means to not being switched in
                            int number;
                            PlayerStat ps = new PlayerStat();
                            number = GenPlayerNumber();
                            if (!GetPlayerData(playercell, true, ref ps))
                            {
                                logger.Error("Failed to get player data.");
                            }
                            md.homeplayers.Add(number, ps);
                        }
                    }
                    tab = tables[1];
                    // Away sub-players
                    for (int i = 0; i < tab.ChildNodes.Count; i++)
                    {
                        HtmlNode playercell = tab.ChildNodes[i].ChildNodes[2];
                        if (IsSwitchOut(playercell.InnerText))
                        {
                            // IsSwitchOut used to check if has time data of the player
                            // Sub without the time data, means to not being switched in
                            int number;
                            PlayerStat ps = new PlayerStat();
                            number = GenPlayerNumber();
                            if (!GetPlayerData(playercell, true, ref ps))
                            {
                                logger.Error("Failed to get player data.");
                            }
                            md.awayplayers.Add(number, ps);
                        }
                    }
                }
                return true;
            }
            catch (Exception ex)
            {
                logger.Error("GetPlayers# Failed: exception");
                logger.Error(ex.ToString());
                return false;
            }
        }

        private bool GetPlayerData(HtmlNode playercell, bool isSub, ref PlayerStat ps)
        {
            try
            {
                ps.posID = PlayerPos.UnknownID;
                ps.isPlayerSub = isSub;
                ps.playerName = GetPlayerName(HtmlEntity.DeEntitize(playercell.SelectSingleNode("./a[1]").InnerText));
                ps.playerID = GetPlayerID(playercell.SelectSingleNode("./a[1]").Attributes["href"].Value);
                if (isSub)
                {
                    ps.isSwitchOut = IsSwitchOut2(playercell.InnerText);
                }
                else
                {
                    ps.isSwitchOut = IsSwitchOut(playercell.InnerText);
                }
                ps.markID = Marking.GetMarkingID(ps.isPlayerSub, ps.isSwitchOut);
                return true;
            }
            catch (Exception ex)
            {
                logger.Error("#GetPlayerData# Failed: Exception");
                logger.Error(ex.ToString());
                return false;
            }
        }

        public string GetPlayerName(string str)
        {
            if (str.IndexOf('(') != -1)
            {
                return str.Substring(0, str.IndexOf('('));
            }
            else
            {
                return str;
            }
        }

        public int GetPlayerID(string str)
        {
            //Format: "http://www.scottishfootballleague.com/player/david-scott/338814/"
            string temp = str.Substring(0, str.Length - 1);
            temp = temp.Substring(temp.LastIndexOf('/') + 1);
            return int.Parse(temp);
        }

        public bool IsSwitchOut(string str)
        {
            if (str.IndexOf('(') != -1)
            {
                return true;
            }
            else
            {
                return false;
            }
        }

        public bool IsSwitchOut2(string str)
        {
            var result = from s in str
                         where s == '('
                         select s;
            if (result.Count() == 2)
            {
                return true;
            }
            else
            {
                return false;
            }
        }

        public bool GetBookings(ref HtmlNode matchRow, bool isYellowCard, ref MatchData md)
        {
            try
            {
                HtmlNode tr;
                if (matchRow.NextSibling != null)
                {
                    tr = matchRow.NextSibling;
                    if (tr.Name == "#text")
                    {
                        if (tr.NextSibling != null)
                        {
                            tr = tr.NextSibling;
                        }
                        else
                        {
                            return true;
                        }
                    }
                }
                else
                {
                    return true;
                }
                if(tr.SelectSingleNode(".//td[@class='stage_round']") != null)
                {
                    return true;
                }
                if (isYellowCard && tr.InnerText.IndexOf("Bookings") != -1)
                {
                    //yellow card
                    HtmlNode tc;
                    tc = tr.SelectSingleNode("./td[1]");//home players
                    if (!string.IsNullOrEmpty(HtmlEntity.DeEntitize(tc.InnerText.Trim())))
                    {
                        for (int i = 0; i < tc.ChildNodes.Count; i++)
                        {
                            if (tc.ChildNodes[i].Name == "#text")
                            {
                                if(!GetPlayerBooking(tc.ChildNodes[i].InnerText, isYellowCard, ref  md.homeplayers))
                                {
                                    return false;
                                }
                                md.match.homeYCard = md.match.homeYCard + 1;
                            }
                        }
                    }
                    tc = tr.SelectSingleNode("./td[3]");//away players
                    if (!string.IsNullOrEmpty(HtmlEntity.DeEntitize(tc.InnerText.Trim())))
                    {
                        for (int i = 0; i < tc.ChildNodes.Count; i++)
                        {
                            if (tc.ChildNodes[i].Name == "#text")
                            {
                                if (!GetPlayerBooking(tc.ChildNodes[i].InnerText, isYellowCard, ref  md.awayplayers))
                                {
                                    return false;
                                }
                                md.match.awayYCard = md.match.awayYCard + 1;
                            }
                        }
                    }
                }

                if (tr.InnerText.IndexOf("Dismissals") != -1)
                {
                    //red card
                    HtmlNode tc;
                    tc = tr.SelectSingleNode("./td[1]");//home players
                    if (!string.IsNullOrEmpty(HtmlEntity.DeEntitize(tc.InnerText.Trim())))
                    {
                        for (int i = 0; i < tc.ChildNodes.Count; i++)
                        {
                            if (tc.ChildNodes[i].Name == "#text")
                            {
                                if (!GetPlayerBooking(tc.ChildNodes[i].InnerText, isYellowCard, ref  md.homeplayers))
                                {
                                    return false;
                                }
                                md.match.homeRCard = md.match.homeRCard + 1;
                            }
                        }
                    }
                    tc = tr.SelectSingleNode("./td[3]");//away players
                    if (!string.IsNullOrEmpty(HtmlEntity.DeEntitize(tc.InnerText.Trim())))
                    {
                        for (int i = 0; i < tc.ChildNodes.Count; i++)
                        {
                            if (tc.ChildNodes[i].Name == "#text")
                            {
                                if (!GetPlayerBooking(tc.ChildNodes[i].InnerText, isYellowCard, ref  md.awayplayers))
                                {
                                    return false;
                                }
                                md.match.awayRCard = md.match.awayRCard + 1;
                            }
                        }
                    }
                }
                matchRow = tr;
                return true;
            }
            catch (Exception ex)
            {
                logger.Error("GetBookings# Failed: exception");
                logger.Error(ex.ToString());
                return false;
            }
        }

        public bool GetPlayerBooking(string str,bool isYellowCard, ref Dictionary<int, PlayerStat> players)
        {
            try
            {
                string temp = str;
                int tempindex;
                tempindex = temp.IndexOf('(');
                string sname = temp.Substring(0, tempindex).Trim();
                var result = from p in players.Values
                             where sname.Equals(this.ShortName(p.playerName))
                             select p;
                if (result.Count() == 1)
                {
                    PlayerStat p = result.First();
                    temp = temp.Substring(tempindex + 1, temp.IndexOf(')') - tempindex - 1);
                    int min;
                    GetBookingMin(temp, out min);
                    if (isYellowCard)
                    {
                        p.yCardMin1 = min;
                    }
                    else
                    {
                        p.rCardMin = min;
                    }
                }
                else
                {
                    throw new Exception("Error in find the booking player. result count is " + result.Count());
                }
                return true;
            }
            catch (Exception ex)
            {
                logger.Error("GetPlayerBooking# Failed: exception");
                logger.Error(ex.ToString());
                return false;
            }
        }

        public bool GetBookingMin(string str,out int min)
        {
            try
            {
                string temp1 = str;
                string temp2;
                if (temp1.IndexOf('+') != -1)
                {
                    temp2 = temp1.Substring(temp1.IndexOf('+') + 1);
                    temp1 = temp1.Substring(0, temp1.IndexOf('+'));
                    min = int.Parse(temp1) + int.Parse(temp2);
                }
                else
                {
                    min = int.Parse(temp1);
                }
                return true;
            }
            catch (Exception ex)
            {
                logger.Error("GetBookingMin# Failed: exception");
                logger.Error(ex.ToString());
                min = -1;
                return false;
            }
        }
        
        public bool GetGoals(ref HtmlNode matchRow, ref MatchData md)
        {
            logger.Debug("GetGoals# Start.");
            try
            {
                HtmlNode tr;
                if (matchRow.NextSibling != null)
                {
                    tr = matchRow.NextSibling;
                }
                else
                {
                    return true;
                }
                while (true)
                {
                    if (tr.Name == "#text" || tr.SelectNodes("./td[@class='colspan']") != null)
                    {
                        tr = tr.NextSibling;
                        if (tr == null)
                        {
                            return true;
                        }
                    }
                    else
                    {
                        break;
                    }
                }
                
                if (tr.InnerText.IndexOf("Bookings") != -1 || tr.InnerText.IndexOf("Dismissals") != -1
                    || tr.SelectSingleNode(".//td[@class='stage_round']") != null)
                {
                    return true;
                }
                //Goal data
                HtmlNode tc;
                tc = tr.SelectSingleNode("./td[1]");//home players
                if (!string.IsNullOrEmpty(HtmlEntity.DeEntitize(tc.InnerText.Trim())))
                {
                    for (int i = 0; i < tc.ChildNodes.Count; i++)
                    {
                        if (tc.ChildNodes[i].Name == "#text")
                        {
                            if (!GetPlayerGoal(tc.ChildNodes[i].InnerText, ref md.homeplayers, ref md.awayplayers))
                            {
                                return false;
                            }
                        }
                    }
                }
                tc = tr.SelectSingleNode("./td[3]");//away players
                if (!string.IsNullOrEmpty(HtmlEntity.DeEntitize(tc.InnerText.Trim())))
                {
                    for (int i = 0; i < tc.ChildNodes.Count; i++)
                    {
                        if (tc.ChildNodes[i].Name == "#text")
                        {
                            if(!GetPlayerGoal(tc.ChildNodes[i].InnerText, ref md.awayplayers, ref md.homeplayers))
                            {
                                return false;
                            }
                        }
                    }
                }
                matchRow = tr;
                logger.Debug("GetGoals# End.");
                return true;
            }
            catch (Exception ex)
            {
                logger.Error("GetGoals# Failed: Exception");
                logger.Error(ex.ToString());
                return false;
            }
        }
        
        public bool GetPlayerGoal(string str, ref Dictionary<int, PlayerStat> players1, ref Dictionary<int, PlayerStat> players2)
        {
            try
            {
                Dictionary<int, PlayerStat> players = null;
                string temp = str;
                int tempindex;
                bool isOwnGoal = false;
                if (str.IndexOf("[OG]") != -1)
                {
                    isOwnGoal = true;
                    temp = str.Remove(0, 4).TrimStart();
                    players = players2;
                }
                else
                {
                    players = players1;
                }

                tempindex = temp.IndexOf('(');
                string sname = temp.Substring(0, tempindex).Trim();
                var result = from p in players.Values
                             where sname.Equals(this.ShortName(p.playerName))
                             select p;
                if (result.Count() == 1)
                {
                    List<MinuteStat> minStat;
                    PlayerStat p = result.First();
                    temp = temp.Substring(tempindex + 1,temp.IndexOf(')') - tempindex - 1);
                    GetGoalsMin(temp,isOwnGoal, out minStat);
                    if (!isOwnGoal)
                    {
                        p.goal = p.goal + minStat.Count;
                        p.MinStat.AddRange(minStat);
                    }
                    else
                    {
                        p.owngoal = p.owngoal + minStat.Count;
                        p.MinStat.AddRange(minStat);
                    }
                }
                else
                {
                    throw new Exception("Error in find the goal player. result count is "+result.Count());
                }
                return true;
            }
            catch (Exception ex)
            {
                logger.Error("GetPlayerGoal# Failed: exception");
                logger.Error(ex.ToString());
                return false;
            }
        }
        
        public bool GetGoalsMin(string str, bool isOwnGoal, out List<MinuteStat> goalMins)
        {
            try
            {
                string[] minlist = str.Split(',');
                goalMins = new List<MinuteStat>();
                for (int i = 0; i < minlist.Length; i++)
                {
                    MinuteStat ms = new MinuteStat();
                    string temp1;
                    string temp2;
                    if (minlist[i].IndexOf("pen") != -1)
                    {
                        temp1 = minlist[i].Replace("pen", "").Trim();
                    }
                    else
                    {
                        temp1 = minlist[i];
                    }
                    if (temp1.IndexOf('+') != -1)
                    {
                        temp2 = temp1.Substring(temp1.IndexOf('+') + 1);
                        temp1 = temp1.Substring(0, temp1.IndexOf('+'));
                        ms.goalMin = int.Parse(temp1) + int.Parse(temp2);
                    }
                    else
                    {
                        ms.goalMin = int.Parse(temp1);
                    }
                    ms.isOwnGoal = isOwnGoal;
                    goalMins.Add(ms);
                }
                return true;
            }
            catch (Exception ex)
            {
                logger.Error("GetGoalsMin# Failed: exception");
                logger.Error(ex.ToString());
                goalMins = null;
                return false;
            }
        }
        
        public string ShortName(string name)
        {
            try
            {
                string first = name.Substring(0, name.IndexOf(' '));
                return string.Format("{0}{1}", first[0], name.Remove(0, first.Length));
            }
            catch (Exception ex)
            {
                logger.ErrorFormat("Cannot get the short name of {0}", name);
                return name;
            }
        }
        
        public int GenPlayerNumber()
        {
            return ((tempNumber++) % 30);
        }
    }
}
